package cz.incad.psh.oai_solr;

import java.io.File;
import java.io.FilenameFilter;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.Reader;
import java.io.StringReader;
import java.io.StringWriter;
import java.io.Writer;
import java.net.HttpURLConnection;
import java.net.ProtocolException;
import java.net.URL;
import java.util.logging.Level;
import java.util.logging.Logger;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.transform.Transformer;
import javax.xml.transform.TransformerFactory;
import javax.xml.transform.dom.DOMSource;
import javax.xml.transform.stream.StreamResult;
import javax.xml.transform.stream.StreamSource;
import javax.xml.xpath.XPath;
import javax.xml.xpath.XPathConstants;
import javax.xml.xpath.XPathExpression;
import javax.xml.xpath.XPathFactory;
import org.apache.commons.io.FileUtils;
import org.w3c.dom.Document;

/**
 *
 * @author Alberto Hernandez
 */
public class Converter {

    private Logger logger = Logger.getLogger(this.getClass().getName());
    URL solrUrl;
    Transformer transformer;
    ProgramArguments arguments;
    int deletes;
    public static int total = 0;
    public static boolean isCancelled = false;
    public static boolean isRunning = false;
    public static float hundred = (float) 100.0;

    public static boolean getIsRunning() {
        return Converter.isRunning;
    }

    public static int getProccessed() throws Exception {
        if (!Converter.isRunning) {
            throw new Exception("not running");
        }
        return Converter.total;
    }

    public static void setCancelled(boolean cancel) {
        Converter.isCancelled = cancel;
    }

    public Converter(ProgramArguments arguments) throws Exception {
        this.arguments = arguments;
        TransformerFactory tfactory = TransformerFactory.newInstance();
        File xslFile = new File(arguments.xsl);
        if(xslFile.exists()){
            StreamSource xslt = new StreamSource(xslFile);
            transformer = tfactory.newTransformer(xslt);
        }else{
            logger.log(Level.WARNING, "File " + xslFile.getAbsolutePath() + " doesn't exists. Using default.");
            InputStream source = this.getClass().getResourceAsStream("res/invenio.xsl");
            System.out.println("this.getClass() -> " + this.getClass().getCanonicalName());
            System.out.println("source -> " + source);
            transformer = tfactory.newTransformer(new StreamSource(source));
        }

        
        solrUrl = new URL(arguments.dest + "/update");
    }

    public String convert() throws Exception {
        try {
            Converter.total = 0;
            Converter.isRunning = true;
            Converter.isCancelled = false;
            deletes = 0;

            factory = XPathFactory.newInstance();
            xpath = factory.newXPath();
            xpath.setNamespaceContext(new OAINamespaceContext());
            
            
        if (arguments.clear) {
            clearIndex();
        }
        
            if (arguments.recordId != null) {
                getRecord(arguments.recordId);
            } else if (arguments.dir != null) {
                fromDir();
            } else {
                harvest();
            }
            String msg;
            if (Converter.isCancelled) {
                msg = "Cancelled. ";
            } else {
                msg = "Finished. ";
            }
            msg += "Indexed " + total + " documents. Deleted " + deletes + " documents. ";
            Converter.isRunning = false;
            logger.info(msg);
            return msg;
        } catch (Exception ex) {
            Converter.isRunning = false;
            throw new Exception(ex);
        }
    }
    XPathFactory factory = XPathFactory.newInstance();
    XPath xpath;
    XPathExpression expr;

    private String xmlToString(Document xmldoc) {
        try {

            StringWriter sw = new StringWriter();
            Transformer serializer = TransformerFactory.newInstance().newTransformer();
            serializer.setParameter("xml-declaration", false);
            serializer.transform(new DOMSource(xmldoc), new StreamResult(sw));
            return sw.toString();


        } catch (Exception ex) {
            logger.severe("Can't convert xml document to string" + ex.toString());
        }
        return "";
    }

    private void getRecord(String id) throws Exception {
        String urlStr = arguments.orig + "?verb=GetRecord&metadataPrefix=" + arguments.metadataPrefix + "&identifier=" + arguments.recordId;
        logger.info("urlStr: " + urlStr);
        java.net.URL url = new java.net.URL(urlStr);
        DocumentBuilderFactory df = DocumentBuilderFactory.newInstance();
        df.setNamespaceAware(true);
        DocumentBuilder builder = df.newDocumentBuilder();
        //builder.
        org.w3c.dom.Document solrDom;
        InputStream is;
        factory = XPathFactory.newInstance();
        xpath = factory.newXPath();
        try {
            is = url.openStream();
            solrDom = builder.parse(is);
        } catch (Exception ex) {
            logger.log(Level.WARNING, "", ex);
            is = url.openStream();
            solrDom = builder.parse(is);
        }

        String xPathStr = "//error/@code";
        expr = xpath.compile(xPathStr);
        String error = (String) expr.evaluate(solrDom, XPathConstants.STRING);
        if (error == null || error.equals("")) {

            StreamResult destStream = new StreamResult(new StringWriter());
            transformer.transform(new DOMSource(solrDom), destStream);
            StringWriter sw = (StringWriter) destStream.getWriter();
            //logger.info(sw.toString());
            postData(new StringReader(sw.toString()), new StringBuilder());
            commit();
        } else {
            xPathStr = "//error/text()";
            expr = xpath.compile(xPathStr);
            String errorText = (String) expr.evaluate(solrDom, XPathConstants.STRING);
            logger.info("Error from OAI response: " + error + " value: " + errorText);
            return;
        }
    }

    private void harvest() throws Exception {
        String urlStr = arguments.orig + "?verb=ListRecords&metadataPrefix=" + arguments.metadataPrefix + "&set=" + arguments.set;
        if (arguments.from != null) {
            urlStr += "&from=" + arguments.from;
        }
        if (arguments.until != null) {
            urlStr += "&until=" + arguments.until;
        }
        logger.info("urlStr: " + urlStr);
        java.net.URL url = new java.net.URL(urlStr);
        DocumentBuilderFactory df = DocumentBuilderFactory.newInstance();
        df.setNamespaceAware(true);

        DocumentBuilder builder = df.newDocumentBuilder();
        org.w3c.dom.Document solrDom;
        InputStream is;
        try {
            is = url.openStream();
        } catch (Exception ex) {
            logger.log(Level.WARNING, "", ex);
            is = url.openStream();
        }
        solrDom = builder.parse(is);
        proccessInput(solrDom);
        boolean hasResuption = true;
        while (hasResuption && !Converter.isCancelled) {//&& x < 10) {

            String xPathStr = "//oai:error/@code";
            expr = xpath.compile(xPathStr);
            String error = (String) expr.evaluate(solrDom, XPathConstants.STRING);

            if (error == null || error.equals("")) {
                String xmlStr = xmlToString(solrDom);
                if (arguments.saveDir != null) {
                    FileUtils.writeStringToFile(new File(arguments.saveDir + total + ".xml"), xmlStr);
                }
                //logger.info(xmlStr);
                String resumptionToken = "";
                int pos1 = xmlStr.indexOf("<resumptionToken");
                if (pos1 > 0) {
                    int pos1a = xmlStr.indexOf(">", pos1);
                    int pos2 = xmlStr.indexOf("</resumptionToken>");
                    resumptionToken = xmlStr.substring(pos1a + 1, pos2);
                }

                if (!resumptionToken.equals("")) {
                    urlStr = arguments.orig + "?verb=ListRecords&resumptionToken=" + resumptionToken;
                    logger.info("urlStr: " + urlStr);
                    url = new java.net.URL(urlStr);
                    InputStream is2;
                    try {
                        is2 = url.openStream();
                    } catch (Exception ex) {
                        logger.log(Level.WARNING, "", ex);
                        is2 = url.openStream();
                    }
                    solrDom = builder.parse(is2);
                    proccessInput(solrDom);



                } else {
                    logger.info("Processed last resumption token ");
                    hasResuption = false;
                    return;
                }
            } else {
                xPathStr = "//error/text()";
                expr = xpath.compile(xPathStr);
                String errorText = (String) expr.evaluate(solrDom, XPathConstants.STRING);
                if (error.contains("noRecordsMatch")) {
                    logger.info("No data in response: " + error + " value: " + errorText + " will continue");
                    return;
                }
                logger.info("Error from OAI response: " + error + " value: " + errorText);
                return;
            }
        }
    }

    private void proccessInput(org.w3c.dom.Document solrDom) throws Exception {
        String xPathStr = "count(//oai:header)";
        expr = xpath.compile(xPathStr);
        int adds = (int) Math.floor((Double) expr.evaluate(solrDom, XPathConstants.NUMBER));

        //deletes
        xPathStr = "count(//oai:header[@status='deleted'])";
        expr = xpath.compile(xPathStr);
        int dels = (int) Math.floor((Double) expr.evaluate(solrDom, XPathConstants.NUMBER));

        StreamResult destStream = new StreamResult(new StringWriter());
        transformer.transform(new DOMSource(solrDom), destStream);
        StringWriter sw = (StringWriter) destStream.getWriter();
        //logger.info(sw.toString());
        postData(new StringReader(sw.toString()), new StringBuilder());
        commit();


        total = total + adds - dels;
        deletes += dels;
        logger.info("Indexed " + total + " documents. Deleted " + deletes + " documents. ");

    }

    private void fromDir() throws Exception {

        FilenameFilter filter = new FilenameFilter() {
            @Override
            public boolean accept(File dir, String name) {
                return name.endsWith(".xml");
            }
        };
        logger.info("Processing files");
        File dir = new File(arguments.dir);
        File[] children = dir.listFiles(filter);
        DocumentBuilderFactory df = DocumentBuilderFactory.newInstance();
        df.setNamespaceAware(true);
        DocumentBuilder builder = df.newDocumentBuilder();
        org.w3c.dom.Document solrDom;
        InputStream is;
        for (int i = 0; i < children.length; i++) {
            logger.info("Processing file " + children[i].getAbsolutePath());
            proccessInput(builder.parse(children[i]));
            logger.info("xml " + children[i].getName() + " processed");
        }
        commit();
    }
    
    

    private void clearIndex() throws Exception {
        String sb = "<delete><query>*:*</query></delete>";
        logger.log(Level.INFO, "clearing index...");
        postData(new StringReader(sb), new StringBuilder());
    }

    private void commit() throws java.rmi.RemoteException, Exception {

        postData(new StringReader("<commit />"), new StringBuilder());

    }

    private void postData(Reader data, StringBuilder output)
            throws Exception {

        HttpURLConnection urlc = null;
        String POST_ENCODING = "UTF-8";
        try {
            urlc = (HttpURLConnection) solrUrl.openConnection();
            try {
                urlc.setRequestMethod("POST");
            } catch (ProtocolException e) {
                throw new Exception("Shouldn't happen: HttpURLConnection doesn't support POST??", e);
            }
            urlc.setDoOutput(true);
            urlc.setDoInput(true);
            urlc.setUseCaches(false);
            urlc.setAllowUserInteraction(false);
            urlc.setRequestProperty("Content-type", "text/xml; charset=" + POST_ENCODING);

            OutputStream out = urlc.getOutputStream();

            try {
                Writer writer = new OutputStreamWriter(out, POST_ENCODING);
                pipe(data, writer);
                writer.close();
            } catch (IOException e) {
                throw new Exception("IOException while posting data", e);
            } finally {
                if (out != null) {
                    out.close();
                }
            }

            InputStream in = urlc.getInputStream();
            int status = urlc.getResponseCode();
            StringBuilder errorStream = new StringBuilder();
            try {
                if (status != HttpURLConnection.HTTP_OK) {
                    errorStream.append("postData URL=").append(arguments.dest).append(" HTTP response code=").append(status).append(" ");
                    throw new Exception("URL=" + arguments.dest + " HTTP response code=" + status);
                }
                Reader reader = new InputStreamReader(in);
                pipeString(reader, output);
                reader.close();
            } catch (IOException e) {
                throw new Exception("IOException while reading response", e);
            } finally {
                if (in != null) {
                    in.close();
                }
            }

            InputStream es = urlc.getErrorStream();
            if (es != null) {
                try {
                    Reader reader = new InputStreamReader(es);
                    pipeString(reader, errorStream);
                    reader.close();
                } catch (IOException e) {
                    throw new Exception("IOException while reading response", e);
                } finally {
                    if (es != null) {
                        es.close();
                    }
                }
            }
            if (errorStream.length() > 0) {
                throw new Exception("postData error: " + errorStream.toString());
            }

        } catch (IOException e) {
            throw new Exception("Solr has throw an error. Check tomcat log. " + e);
        } finally {
            if (urlc != null) {
                urlc.disconnect();
            }
        }
    }

    /**
     * Pipes everything from the reader to the writer via a buffer
     */
    private static void pipe(Reader reader, Writer writer) throws IOException {
        char[] buf = new char[1024];
        int read = 0;
        while ((read = reader.read(buf)) >= 0) {
            writer.write(buf, 0, read);
        }
        writer.flush();
    }

    /**
     * Pipes everything from the reader to the writer via a buffer except lines
     * starting with '<?'
     */
    private static void pipeString(Reader reader, StringBuilder writer) throws IOException {
        char[] buf = new char[1024];
        int read = 0;
        while ((read = reader.read(buf)) >= 0) {
            if (!(buf[0] == '<' && buf[1] == '?')) {
                writer.append(buf, 0, read);
            }
        }
    }
}
